# -*- coding:utf-8 -*-
# @Time : 2021/2/22 11:38
# @Author : Administrator
# @File : doubandushu.py
# @Software: PyCharm
# @Motto: good good study,day day up


import requests
from parsel import Selector
import pymongo
from loguru import logger

myclient = pymongo.MongoClient("mongodb://localhost:27017/")
mydb = myclient["spiders"]
mycol = mydb["doubandushu"]

for page in range(0, 250, 25):
    url = f'https://book.douban.com/top250?start={page}'
    logger.info(f'正在获取{url}网页的数据.......')
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36'
    }
    response = requests.get(url, headers=headers)
    selector = Selector(response.text)
    books = selector.xpath("//table")
    num = 0
    for book in books:
        num += 1
        book_name = book.xpath('.//div[@class="pl2"]//a//@title').extract_first()
        author = book.xpath('.//p[@class="pl"]//text()').extract_first()
        book_link = book.xpath('.//div[@class="pl2"]//a//@href').extract_first()
        book_score = book.xpath('.//span[@class="rating_nums"]//text()').extract_first()
        brief = book.xpath('.//span[@class="inq"]//text()').extract_first()
        item = {
            # '_id': num,
            'book_name': book_name,
            'author': author,
            'book_link': book_link,
            'book_score': book_score,
            'brief': brief
        }
        # items = items.append(item)
        # print(items)
        # mycol.insert_one(items)
        logger.info(item)
        logger.info('正在写入mongo数据库........')
        mycol.insert_one(item)
        logger.info('写入完成.....')
